{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Requirement already satisfied: knext in /raid/users/kag/Desktop/.venv/lib/python3.11/site-packages (1.2.1)\n",
      "Requirement already satisfied: networkx in /raid/users/kag/Desktop/.venv/lib/python3.11/site-packages (from knext) (3.4.2)\n",
      "Requirement already satisfied: Click in /raid/users/kag/Desktop/.venv/lib/python3.11/site-packages (from knext) (8.1.7)\n",
      "Requirement already satisfied: pandas in /raid/users/kag/Desktop/.venv/lib/python3.11/site-packages (from knext) (2.2.3)\n",
      "Requirement already satisfied: typer in /raid/users/kag/Desktop/.venv/lib/python3.11/site-packages (from knext) (0.15.2)\n",
      "Requirement already satisfied: requests in /raid/users/kag/Desktop/.venv/lib/python3.11/site-packages (from knext) (2.32.3)\n",
      "Requirement already satisfied: pathlib in /raid/users/kag/Desktop/.venv/lib/python3.11/site-packages (from knext) (1.0.1)\n",
      "Requirement already satisfied: pytest in /raid/users/kag/Desktop/.venv/lib/python3.11/site-packages (from knext) (6.2.5)\n",
      "Requirement already satisfied: numpy>=1.23.2 in /raid/users/kag/Desktop/.venv/lib/python3.11/site-packages (from pandas->knext) (1.26.4)\n",
      "Requirement already satisfied: python-dateutil>=2.8.2 in /raid/users/kag/Desktop/.venv/lib/python3.11/site-packages (from pandas->knext) (2.8.2)\n",
      "Requirement already satisfied: pytz>=2020.1 in /raid/users/kag/Desktop/.venv/lib/python3.11/site-packages (from pandas->knext) (2024.2)\n",
      "Requirement already satisfied: tzdata>=2022.7 in /raid/users/kag/Desktop/.venv/lib/python3.11/site-packages (from pandas->knext) (2024.2)\n",
      "Requirement already satisfied: attrs>=19.2.0 in /raid/users/kag/Desktop/.venv/lib/python3.11/site-packages (from pytest->knext) (24.2.0)\n",
      "Requirement already satisfied: iniconfig in /raid/users/kag/Desktop/.venv/lib/python3.11/site-packages (from pytest->knext) (2.1.0)\n",
      "Requirement already satisfied: packaging in /raid/users/kag/Desktop/.venv/lib/python3.11/site-packages (from pytest->knext) (24.1)\n",
      "Requirement already satisfied: pluggy<2.0,>=0.12 in /raid/users/kag/Desktop/.venv/lib/python3.11/site-packages (from pytest->knext) (1.5.0)\n",
      "Requirement already satisfied: py>=1.8.2 in /raid/users/kag/Desktop/.venv/lib/python3.11/site-packages (from pytest->knext) (1.11.0)\n",
      "Requirement already satisfied: toml in /raid/users/kag/Desktop/.venv/lib/python3.11/site-packages (from pytest->knext) (0.10.2)\n",
      "Requirement already satisfied: charset-normalizer<4,>=2 in /raid/users/kag/Desktop/.venv/lib/python3.11/site-packages (from requests->knext) (3.4.0)\n",
      "Requirement already satisfied: idna<4,>=2.5 in /raid/users/kag/Desktop/.venv/lib/python3.11/site-packages (from requests->knext) (3.10)\n",
      "Requirement already satisfied: urllib3<3,>=1.21.1 in /raid/users/kag/Desktop/.venv/lib/python3.11/site-packages (from requests->knext) (2.2.3)\n",
      "Requirement already satisfied: certifi>=2017.4.17 in /raid/users/kag/Desktop/.venv/lib/python3.11/site-packages (from requests->knext) (2024.8.30)\n",
      "Requirement already satisfied: typing-extensions>=3.7.4.3 in /raid/users/kag/Desktop/.venv/lib/python3.11/site-packages (from typer->knext) (4.12.2)\n",
      "Requirement already satisfied: shellingham>=1.3.0 in /raid/users/kag/Desktop/.venv/lib/python3.11/site-packages (from typer->knext) (1.5.4)\n",
      "Requirement already satisfied: rich>=10.11.0 in /raid/users/kag/Desktop/.venv/lib/python3.11/site-packages (from typer->knext) (13.9.3)\n",
      "Requirement already satisfied: six>=1.5 in /raid/users/kag/Desktop/.venv/lib/python3.11/site-packages (from python-dateutil>=2.8.2->pandas->knext) (1.16.0)\n",
      "Requirement already satisfied: markdown-it-py>=2.2.0 in /raid/users/kag/Desktop/.venv/lib/python3.11/site-packages (from rich>=10.11.0->typer->knext) (3.0.0)\n",
      "Requirement already satisfied: pygments<3.0.0,>=2.13.0 in /raid/users/kag/Desktop/.venv/lib/python3.11/site-packages (from rich>=10.11.0->typer->knext) (2.18.0)\n",
      "Requirement already satisfied: mdurl~=0.1 in /raid/users/kag/Desktop/.venv/lib/python3.11/site-packages (from markdown-it-py>=2.2.0->rich>=10.11.0->typer->knext) (0.1.2)\n",
      "Note: you may need to restart the kernel to use updated packages.\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "평가 진행 중:  46%|████▌     | 206/450 [2:22:10<3:06:07, 45.77s/it]"
     ]
    }
   ],
   "source": [
    "%pip install knext\n",
    "import os\n",
    "import json\n",
    "import pandas as pd\n",
    "from tqdm import tqdm\n",
    "\n",
    "# EVA 평가를 위한 모듈 임포트 (evaFor2wiki.py와 동일한 Evaluate 사용)\n",
    "from kag.common.benchmarks.evaluate import Evaluate\n",
    "\n",
    "# LangChain 관련\n",
    "from langchain_huggingface import HuggingFaceEmbeddings  # 업데이트된 패키지 사용\n",
    "from langchain_text_splitters import RecursiveCharacterTextSplitter\n",
    "from langchain_community.vectorstores.faiss import FAISS\n",
    "from langchain.schema import Document\n",
    "from langchain_core.prompts import ChatPromptTemplate\n",
    "from langchain_core.messages import HumanMessage\n",
    "\n",
    "# EnsembleRetriever, KiwiBM25Retriever 추가\n",
    "from langchain.retrievers import EnsembleRetriever\n",
    "from langchain_teddynote.retrievers import KiwiBM25Retriever\n",
    "\n",
    "# deeoseek_r1 모델 불러오기 (deeoseek_r1_model_load.py에 정의되어 있다고 가정)\n",
    "from ollama_model_load import deepseek_r1\n",
    "\n",
    "os.environ[\"CUDA_VISIBLE_DEVICES\"] = \"0\"\n",
    "\n",
    "\n",
    "# 1) RAG 검색 수행 함수 (EnsembleRetriever 적용)\n",
    "def perform_rag(\n",
    "    question: str,\n",
    "    text_file: str = \"2wiki_corpus.json\",\n",
    "    chunk_size: int = 100,\n",
    "    chunk_overlap: int = 50,\n",
    "    device: str = \"cuda\"\n",
    ") -> str:\n",
    "    with open(text_file, \"r\", encoding=\"utf-8\") as f:\n",
    "        full_text = f.read()\n",
    "\n",
    "    splitter = RecursiveCharacterTextSplitter(chunk_size=chunk_size, chunk_overlap=chunk_overlap)\n",
    "    chunks = [Document(page_content=t) for t in splitter.split_text(full_text)]\n",
    "\n",
    "    model_kwargs = {\"device\": device}\n",
    "    encode_kwargs = {\"normalize_embeddings\": True}\n",
    "    embeddings = HuggingFaceEmbeddings(\n",
    "        model_name=\"intfloat/multilingual-e5-large\",\n",
    "        model_kwargs=model_kwargs,\n",
    "        encode_kwargs=encode_kwargs,\n",
    "    )\n",
    "\n",
    "    # FAISS Retriever 생성\n",
    "    db_faiss = FAISS.from_documents(chunks, embedding=embeddings)\n",
    "    faiss_retriever = db_faiss.as_retriever(search_kwargs={\"k\": 2})\n",
    "\n",
    "    # KiwiBM25Retriever 생성\n",
    "    kiwi_bm25_retriever = KiwiBM25Retriever.from_documents(chunks)\n",
    "\n",
    "    # EnsembleRetriever 생성\n",
    "    retriever = EnsembleRetriever(\n",
    "        retrievers=[kiwi_bm25_retriever, faiss_retriever],\n",
    "        weights=[0.5, 0.5],\n",
    "        search_type=\"mmr\",\n",
    "    )\n",
    "\n",
    "    context_docs = retriever.invoke(question)\n",
    "    context_text = \"\\n\\n\".join(doc.page_content for doc in context_docs)\n",
    "    return context_text\n",
    "\n",
    "\n",
    "# 2) LLM 질의 함수 (RAG Prompt 포함) - deeoseek_r1 모델 사용\n",
    "def query_llm(context: str, question: str) -> str:\n",
    "    RAG_PROMPT_TEMPLATE = \"\"\"\n",
    "    아래 정보(context)를 참고하여 사용자 질문에 답해주세요:\n",
    "    {context}\n",
    "\n",
    "    질문:\n",
    "    {question}\n",
    "\n",
    "    답변 시, 질문의 핵심만 파악하여 간결하게 1~2문장으로 답변하고, \n",
    "    불필요한 설명은 피합니다.\n",
    "\n",
    "    답변:\n",
    "    \"\"\"\n",
    "    prompt = ChatPromptTemplate.from_template(RAG_PROMPT_TEMPLATE)\n",
    "    formatted_prompt = prompt.format(context=context, question=question)\n",
    "    message = HumanMessage(content=formatted_prompt)\n",
    "\n",
    "    response = deepseek_r1.invoke([message])\n",
    "    return response.content.strip()\n",
    "\n",
    "\n",
    "# 3) 전체 평가 함수: JSON 파일(\"2wiki_qa.json\")에서 _id, type, question, answer를 읽고 EVA 평가(em, f1, answer_similarity)를 계산 후 CSV에 기록\n",
    "def evaluate_model_responses(\n",
    "    json_file: str = \"2wiki_qa.json\",\n",
    "    text_file: str = \"2wiki_corpus.json\",\n",
    "    output_file: str = \"2wiki_result_rag.csv\",\n",
    "    batch_size: int = 5,\n",
    "    chunk_size: int = 100,\n",
    "    chunk_overlap: int = 50,\n",
    "    device: str = \"cuda\"\n",
    "):\n",
    "    processed_count = 0\n",
    "    if os.path.exists(output_file):\n",
    "        try:\n",
    "            existing_df = pd.read_csv(output_file, encoding='utf-8-sig')\n",
    "        except Exception as e:\n",
    "            print(f\"출력 파일 읽기 오류: {e}\")\n",
    "            existing_df = None\n",
    "\n",
    "        if existing_df is not None and not existing_df.empty:\n",
    "            if \"전체 평균\" in str(existing_df.iloc[-1, 0]):\n",
    "                processed_count = len(existing_df) - 1\n",
    "                existing_df = existing_df.iloc[:-1]\n",
    "                existing_df.to_csv(output_file, index=False, encoding='utf-8-sig')\n",
    "            else:\n",
    "                processed_count = len(existing_df)\n",
    "\n",
    "    with open(json_file, \"r\", encoding=\"utf-8\") as f:\n",
    "        qa_list = json.load(f)\n",
    "\n",
    "    total_rows = len(qa_list)\n",
    "    if processed_count >= total_rows:\n",
    "        print(\"이미 모든 행이 처리되었습니다.\")\n",
    "        return pd.read_csv(output_file, encoding='utf-8-sig')\n",
    "\n",
    "    evaluation_results = []\n",
    "    em_list, f1_list, ans_sim_list = [], [], []\n",
    "    evalObj = Evaluate()\n",
    "\n",
    "    for idx in tqdm(range(processed_count, total_rows), desc=\"평가 진행 중\"):\n",
    "        sample = qa_list[idx]\n",
    "        _id = sample[\"_id\"]\n",
    "        type_field = sample[\"type\"]\n",
    "        question = sample[\"question\"]\n",
    "        gold = sample[\"answer\"]\n",
    "\n",
    "        context = perform_rag(\n",
    "            question=question,\n",
    "            text_file=text_file,\n",
    "            chunk_size=chunk_size,\n",
    "            chunk_overlap=chunk_overlap,\n",
    "            device=device\n",
    "        )\n",
    "        generated_response = query_llm(context, question)\n",
    "\n",
    "        metrics = evalObj.getBenchMark([generated_response], [gold])\n",
    "        em_val, f1_val, ans_sim_val = metrics[\"em\"], metrics[\"f1\"], metrics[\"answer_similarity\"]\n",
    "\n",
    "        em_list.append(em_val)\n",
    "        f1_list.append(f1_val)\n",
    "        ans_sim_list.append(ans_sim_val)\n",
    "\n",
    "        evaluation_results.append({\n",
    "            \"_id\": _id,\n",
    "            \"type\": type_field,\n",
    "            \"question\": question,\n",
    "            \"answer\": gold,\n",
    "            \"model_response\": generated_response,\n",
    "            \"em\": em_val,\n",
    "            \"f1\": f1_val,\n",
    "            \"answer_similarity\": ans_sim_val\n",
    "        })\n",
    "\n",
    "        if (len(evaluation_results) % batch_size == 0) or (idx == total_rows - 1):\n",
    "            partial_df = pd.DataFrame(evaluation_results)\n",
    "            partial_df.to_csv(\n",
    "                output_file,\n",
    "                mode='a' if os.path.exists(output_file) and processed_count > 0 else 'w',\n",
    "                index=False,\n",
    "                header=not (os.path.exists(output_file) and processed_count > 0),\n",
    "                encoding='utf-8-sig'\n",
    "            )\n",
    "            evaluation_results = []\n",
    "            processed_count = idx + 1\n",
    "\n",
    "    avg_em = sum(em_list) / len(em_list) if em_list else 0\n",
    "    avg_f1 = sum(f1_list) / len(f1_list) if f1_list else 0\n",
    "    avg_ans_sim = sum(ans_sim_list) / len(ans_sim_list) if ans_sim_list else 0\n",
    "\n",
    "    summary_row = {\n",
    "        \"_id\": \"전체 평균\",\n",
    "        \"type\": \"\",\n",
    "        \"question\": \"\",\n",
    "        \"answer\": \"\",\n",
    "        \"model_response\": \"\",\n",
    "        \"em\": avg_em,\n",
    "        \"f1\": avg_f1,\n",
    "        \"answer_similarity\": avg_ans_sim\n",
    "    }\n",
    "\n",
    "    final_df = pd.read_csv(output_file, encoding='utf-8-sig')\n",
    "    final_df = pd.concat([final_df, pd.DataFrame([summary_row])], ignore_index=True)\n",
    "    final_df.to_csv(output_file, index=False, encoding='utf-8-sig')\n",
    "\n",
    "    print(f\"평가 완료! 결과는 '{output_file}'에 저장되었습니다.\")\n",
    "    return final_df\n",
    "\n",
    "# 4) 메인 실행 예시\n",
    "if __name__ == \"__main__\":\n",
    "    final_df = evaluate_model_responses(\n",
    "        json_file=\"2wiki_qa.json\",       # 2wiki_qa.json 파일 사용\n",
    "        text_file=\"2wiki_corpus.json\",    \n",
    "        output_file=\"2wiki_result_rag.csv\", \n",
    "        batch_size=5,\n",
    "        chunk_size=100,\n",
    "        chunk_overlap=50,\n",
    "        device=\"cuda\"\n",
    "    )\n",
    "    print(final_df.head())\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": ".venv",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.11.0"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}
